#!/usr/local/bin/ruby
# 
#  morphoAnalysis.rb
#   perform MDL-based morphological analysis for each of the ranked lists
#  
#  2007-10-30
#  
#  Indra Neil Sarkar, PhD
#  MBLWHOI Library
#  Marine Biological Laboratory
#  7 MBL Street
#  Woods Hole, MA 02543 USA
#  sarkar@mbl.edu
#  
#  
#  Copyright 2007 Marine Biological Laboratory. All rights reserved.
# 

# ====================================================
# = calculate the raw suffix score (Robustness - DL) =
# ====================================================
def getRawSuffixScore(wordHash, suffix)
  
  # [ROBUSTNESS] - [DL] 
  # [S(F-1) + F(S-1)] - [F + Log(Wc/Wf)/Log(2)]
  
  # calculate the average stem length
  suffixStems = wordHash.keys.grep(/#{suffix}$/)
  totalStemLength = 0
  suffixStems.each do |stem|
    totalStemLength += stem.size - suffix.size
  end
  
  # only do calculations for stems that are greater than 0 (i.e., ignore NULL suffixes)
  # return the rawSuffixScore if greater than 0 (otherwise, return 0)
  if suffixStems.size > 0
    avgStemLength = totalStemLength / suffixStems.size
  
    # count number of words with suffix in wordHash
    suffixCount = wordHash.keys.grep(/#{suffix}$/).size
 
    # calculate descriptiveLength
    descriptiveLength = suffix.size + Math.log(wordHash.size/suffixCount)/Math.log(2)
  
    # calculate robustness
    robustness = (avgStemLength * (suffix.size-1)) + (suffix.size * (avgStemLength-1))
  
    # calculate the rawSuffixScore
    rawSuffixScore = robustness - descriptiveLength
  
    # return the rawSuffixScore (robustness-descriptiveLength)
    return rawSuffixScore if rawSuffixScore > 0  
  else
    return 0
  end
  
  # return 0 by default
  return 0
end

# ========================================================================================
# = determine suffixes based on determined morpheme parts using Harris Succession method =
# ========================================================================================
def suffixAnalysis(fullWordListHash, individualWordListHash)
  
  # suffix hash is where the suffixes will be stored and returned
  suffixHash = Hash.new
  testedSuffixHash = Hash.new
  
  # keep track of processed stems
  testedStemHash   = Hash.new
  
  # go through each word for the full wordList
  wordCount = 0
  fullWordListHash.sort.each do |word,val|
    
    wordTimeStart = Time.new
    # puts word
    # puts word.size
    
    # use successor frequency to determine stem & suffix morphemes
    lastLastFrequency = 0
    lastFrequency = 0
    currentFrequency = 0
    lastCandidateStem = ""
    lastCandidateSuffix = ""
    lastCandidateSuffixList = Hash.new
    
    # consider all possible stem lengths
    for stemLength in 1..(word.size)
      
      
      # split the word into candidate Stem and Suffix morphemes
      word[/(.{#{stemLength}})(.*)/]
      candidateStem = $1
      candidateSuffix = $2
      
      if !testedStemHash.has_key? candidateStem
        if !testedSuffixHash.has_key? candidateSuffix


            candidateSuffixWordList = fullWordListHash.keys.grep(/^#{candidateStem}.+/)

            # calculate the successor frequency at current stem position
            freqCharHash = Hash.new
            candidateSuffixList = Hash.new
            
            candidateSuffixWordList.each do |suffixWord|
              firstChar = suffixWord[stemLength,1]
              freqCharHash[firstChar] = 1
              candidateSuffixList[suffixWord[stemLength,word.size]] = 1
            end
            freqChar = freqCharHash.size
      
            # update frequency values, propogating back values
            lastLastFrequency = lastFrequency
            lastFrequency = currentFrequency
            currentFrequency = freqCharHash.size
            lastCandidateSuffixList = candidateSuffixList
            
            #puts "#{candidateStem} | #{candidateSuffix}---> #{lastLastFrequency} - #{lastFrequency} - #{currentFrequency}"
            
            # determine if morpheme is a suffix candidate using pattern 1-1-1 -- also require suffix size to be longer than one character
            if lastLastFrequency == 1 && lastFrequency == 1 && currentFrequency == 1 && candidateSuffix.size > 1
              #puts "#{word}  **** MORPHEME BREAK: " + candidateStem + ' / ' + candidateSuffix
              suffixHash[lastCandidateSuffix] = 1
              #puts "*** +New suffix: #{candidateSuffix}"
              
              File.open("../morphanalysis/morphemeSuffixList.txt", "a") do |suffixListFile|
                suffixListFile.puts "#{candidateSuffix}"
              end
              
            end
          testedSuffixHash[candidateSuffix] = 1
        end
        testedStemHash[candidateStem] = 1
      end
      
      # propogate back the stem and suffix morphemes
      lastCandidateStem = candidateStem
      lastCandidateSuffix = candidateSuffix
      # lastCandidateSuffixList.clear if lastCandidateSuffixList != nil
      # lastCandidateSuffixList = candidateSuffixList
              
    end
    
    wordTimeEnd = Time.new
    wordTimeDiff = wordTimeEnd - wordTimeStart
    wordCount += 1
    puts  "getting suffixes - (" + ((wordCount.to_f/(fullWordListHash.size).to_f)*100).to_s + '% complete) [' + suffixHash.size.to_s + ' suffixes found] [' + testedSuffixHash.size.to_s + ' morphemes tested] - ' + wordTimeDiff.to_s + "s   -- #{word}"

  end

  return suffixHash
end

# ================================================
# = calculate statistical ranks of the suffixes  =
# ================================================
def rankSuffixes(suffixHash,fullWordListHash,individualWordListHash)
  
  # instantiate rawScoreHash, which will store the rawScores for each word list
  rawScoreHash = Hash.new { |suffix, wordList| suffix[wordList] = {} }
  highScoreHash = Hash.new
  
  ####
  # calculate rawSuffixScore for each suffix, for each list
  suffixCount = 0
  suffixHash.keys.sort.each do |suffix|
    # puts "  >>- Suffix --> #{suffix}"
    
    # calculate the raw suffix score for the complete list of words         
    rawSuffixScore = getRawSuffixScore(fullWordListHash,suffix)
    rawScoreHash[suffix]["!overall"] = rawSuffixScore
    
    # store highest overall score
    if highScoreHash.has_key? "!overall"
      highScoreHash["!overall"] = rawSuffixScore if highScoreHash["!overall"] < rawSuffixScore
    else
      highScoreHash["!overall"] = rawSuffixScore
    end
    
    # puts "overall rawSuffixScore--> #{rawSuffixScore} -- #{highScoreHash['_overall']}"
  
    # calculate the raw suffix score for each of the individual word lists
    individualWordListHash.keys.sort.each do |wordList|
      rawSuffixScore = getRawSuffixScore(individualWordListHash[wordList],suffix)
      rawScoreHash[suffix][wordList] = rawSuffixScore
      
      # store highest score for each word list
      if highScoreHash.has_key? wordList
        highScoreHash[wordList] = rawSuffixScore if highScoreHash[wordList] < rawSuffixScore
      else
        highScoreHash[wordList] = rawSuffixScore
      end
      
      # puts "#{wordList} rawSuffixScore--> #{rawSuffixScore} -- #{highScoreHash[wordList]}"
    end  
             
    suffixCount += 1
    puts  'calculating suffix rank scores - (' + ((suffixCount.to_f/(suffixHash.size).to_f)*100).to_s + '% complete)'
    
  end
  
  return rawScoreHash, highScoreHash
  

end

# =====================================
# = load word list hashes from a file =
# =====================================
def loadWordList(fullWordListHash, individualWordListHash, wordListFileName, wordListName)
  
  # load wordHash with words from wordListFile
  puts "loading #{wordListName}"
  File.open(wordListFileName).each do |word|
    
    word.chomp!
    
    # skip words that have non-alpha characters
    next if word[/[^A-Za-z ]/]
    next if word.size < 2
    
    # load both individual word list and full word list
    individualWordListHash[wordListName][word] = 1  
    fullWordListHash[word] = 1  
  end  
end


####################
####################


timeStart = Time.new


File.open("../morphanalysis/morphemeSuffixList.txt", "w") do |suffixListFile|
  suffixListFile.puts "SUFFIX_MORPHEME"
end

# instantiate the individualWordList hash and the fullWordListHash
individualWordListHash = Hash.new { |wordListName, word| wordListName[word] = {} }
fullWordListHash = Hash.new

# load the hashes from the word list files
loadWordList(fullWordListHash, individualWordListHash, "../morphanalysis/rankLists/phylum.rankList.txt" , "1_phylum")
loadWordList(fullWordListHash, individualWordListHash, "../morphanalysis/rankLists/class.rankList.txt"  , "2_class")
loadWordList(fullWordListHash, individualWordListHash, "../morphanalysis/rankLists/order.rankList.txt"  , "3_order")
loadWordList(fullWordListHash, individualWordListHash, "../morphanalysis/rankLists/family.rankList.txt" , "4_family")
loadWordList(fullWordListHash, individualWordListHash, "../morphanalysis/rankLists/genus.rankList.txt"  , "5_genus")
loadWordList(fullWordListHash, individualWordListHash, "../morphanalysis/rankLists/species.rankList.txt", "6_species")

# perform morphological analysis on full words to determine morphemes
suffixHash = Hash.new
suffixHash = suffixAnalysis(fullWordListHash, individualWordListHash)


# determine rank scores for each of the suffixes
rawScoreHash = Hash.new
highScoreHash = Hash.new
rawScoreHash, highScoreHash = rankSuffixes(suffixHash,fullWordListHash,individualWordListHash)

puts
puts
timeEnd = Time.new
timeDifference = timeEnd - timeStart
puts "Time Start: #{timeStart}"
puts "Time End  : #{timeEnd}"
puts "Time Diff : #{timeDifference/3600} hrs "

# print out matrix
puts
puts "printing out score matrix"
matrixFileName = '../morphanalysis/suffixRankScores.txt'
File.open(matrixFileName, "w") do |matrixFile|

  # print out header line for score matrix
  matrixFile.print "|_overall"
  individualWordListHash.keys.sort.each do |wordListName|
    matrixFile.print "|#{wordListName}"
  end
  matrixFile.print "\n"

  # print out each suffix row of scores, ordered by word lists
  rawScoreHash.keys.sort.each do |suffix|
    if rawScoreHash[suffix]["!overall"] > 0
      matrixFile.print suffix
      rawScoreHash[suffix].keys.sort.each do |wordList|
        #puts "  #{wordList}"
        normalizedScore = rawScoreHash[suffix][wordList] / highScoreHash[wordList]
        # print "|#{rawScoreHash[suffix][wordList]}"
        matrixFile.print "|#{normalizedScore}"
      end
      matrixFile.print "\n"
    end
  end
end

